<?php

require_once("database.php");

// Builds the table of cities from a url on craiglist containing all cities and links to them.
function retrieve($cities) {
	$content = file_get_contents($cities);
	
	$regex = '/<div id="list">(.+?)<\/div>/si';
	preg_match_all($regex, $content, $links);
	
	$list = $links[1][0];
	$regex = '/<a href=\"(.+?)">(<b>)?(.+?)(<\/b>)?<\/a>/';
	preg_match_all($regex, $list, $all);
  
  for($i = 0; $i < count($all[1]); $i+=1){
    $insert_array = array();
    $insert_array['url'] = $all[1][$i] . 'apa/';
    $insert_array['name'] = $all[3][$i];
    query_insert('city', $insert_array);
  }
	
}

# Give an id that corresponds to the city table
function scrape($city_id, $webpage, $today, $page){

  $content = file_get_contents($webpage . 'index' . $page . '.html');
  
  if(!$content){
    $reached_end = true;
  }
  else {
    $content_array = explode("\n",$content);
    
    $reached_end = false;
    
    foreach($content_array as $line){
      if(substr($line,0,3) == "<p>" or substr($line,0,4) == "<h4>"){
        
        // Apartment regex
        // Group 0 matches if it is valid
        // Group 1 matches url
        // Group 2 matches price
        // Group 3 matches bedrooms
        // Group 4 matches description
        // Group 5 matches location
        $apa_regex = '/<a href="\/apa\/(.+?)">(?:\$(\d+))?(?:[^\d]*?(\d) ?(?:br|bdrm|bdr))?(?: - )?(.+?)(?: -)?<\/a><font size="-1"> \((.+)\)<\/font>/';
        
        // Date regex
        // Group 0 matches if it is valid
        // Group 1 matches date
        $date_regex = '/<h4>(.+?)<\/h4>/';
        
        preg_match_all($apa_regex,$line,$apa_matches);
        preg_match_all($date_regex,$line,$date_matches);
        
        // If it matches apartment record and the price is not null
        if(count($apa_matches[0]) > 0 and $apa_matches[2][0] != ""){
        
          // Check to see if entry is already in the table by querying for the url of the entry
          $check = query_select("
            SELECT 1
            FROM apartment
            WHERE url = '" . sql_escape($webpage . $apa_matches[1][0]) . "'"
          );
          
          // It is
          if(count($check) == 0){
          
            // Enter into table
            $apartment_insert = array();
            $apartment_insert['city_id'] = $city_id;
            $apartment_insert['date'] = time();
            $apartment_insert['url'] = $webpage . $apa_matches[1][0];
            $apartment_insert['price'] = (int) $apa_matches[2][0];
            $apartment_insert['bedrooms'] = (int) $apa_matches[3][0];
            $apartment_insert['title'] = $apa_matches[4][0];
            $apartment_insert['location'] = $apa_matches[5][0];
            
            query_insert("apartment",$apartment_insert);
          }
          // It is not
          else {
            // Update the timestamp of the record
            query_update("
              UPDATE apartment
              SET date = " . time() . "
              WHERE url = '" . sql_escape($webpage . $apa_matches[1][0]) . "'"
            );
          }
        }
        
        // If it matches the date record and the date isn't null (this can happen for some reason)
        else if(count($date_matches) > 0 and isset($date_matches[1][0]) and $date_matches[1][0] != ""){
          // If we're on another day, STOP
          
          if($date_matches[1][0] != $today){
            $reached_end = true;
            break; // DROP OUT OF FOR LOOP
          }
          
        }
        
      }
    }
  }
  
  // Set timers
  sleep(11); // Really, seriously, the wait time needs to be at least 10 seconds.
  set_time_limit(30000);
  
  // Recursive call to get the next pages
  if(!$reached_end){
    scrape($city_id, $webpage, $today, $page + 100);
  }
  
}

// Run functionality
	
// If sent with request to update all the cities
// TOO DANGEROUS, don't use more than once.
if($_GET and isset($_GET['cities'])){
  echo 'Scraping cities';
  query_update("DELETE FROM city");
  $urls = retrieve("http://geo.craigslist.org/iso/us");
}
if($_GET and isset($_GET['apartments'])){
  echo 'Scraping apartments';
  $cities = query_select("
    SELECT *
    FROM city
  ");
  foreach($cities as $city){
    $date_to_scrape = date("D M j", time());
    scrape($city['id'], $city['url'], $date_to_scrape, 0);
  }
}

?>
